package mblog.web.job;

import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.springframework.stereotype.Service;
import mtons.modules.utils.PreviewTextUtils;

@Service
public class XiNiuGuShi {
	private static String READ_FILE_NAME = "toutiao.txt";// 简书
	private static String SITE = "http://toutiao.com/";// 简书
	private static String NOW_PAGE_URL_FILE = "XiNiuGuShiPageURL.txt";// 当前页url";
	private static String LUOSH_SPIDER_LOG = "D:/luoshSpiderLog/"; // "D:/tysSpiderLog/";

	@Test
	public void processFace() {
		String url = "";
		// 获取上一次读取页面，继续开始
		String nowPageUrl = ProcessUtil.getNowPageUrl();
		if (StringUtils.isBlank(nowPageUrl)) {
			url = SITE + "/news_hot";
		} else {
			url = nowPageUrl;
		}
		String tid = getArcticleId(url);
		if ("-1".equals(tid)) {
			processMutil(url);
		} else {
			processOne(url);
		}
		// 获取下一页内容的请求URL
		String loadNextPageUrl = getLoadNextPageUrl(url);
		loadNextPageUrl.substring(loadNextPageUrl.indexOf("note_ids", 20));
		// 记录当前读取的页数URL
		nowPageUrl(loadNextPageUrl);
		processNextPage(loadNextPageUrl);
		System.out.println("over------------------------------------");
	}

	// 处理下一页
	public void processNextPage(String loadNextPageUrl) {
		int pageNo = 1;
		String url = loadNextPageUrl;
		String tid = getArcticleId(url);
		if ("-1".equals(tid)) {
			processMutil(url);
		} else {
			processOne(url);
		}
		System.out.println("处理第" + pageNo++ + "页开始！");
		processFace();
	}

	public String getLoadNextPageUrl(String url) {
		String nextPageUrl = SITE;
		try {
			Document doc = ProcessUtil.getDoc(url);
			nextPageUrl += doc.body().select("button.ladda-button ").get(0).attr("data-url");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return nextPageUrl;
	}

	public void processMutil(String url) {
		try {
			Document doc = ProcessUtil.getDoc(url);
			// System.out.println(doc.body());
			Elements links = doc.body().select("div.container");
			for (Element link : links) {
				link.select("em").remove();
				Elements alinks = link.select("a");
				// System.out.println(alinks.get(0));
				String alink = alinks.get(0).absUrl("href");
				System.out.println("开始抓取：" + alink);
				processOne(alink);
			}

		} catch (Exception e) {
		}

	}

	public void processOne(String url) {
		try {
			// http://www.jianshu.com/p/423cbbe0bf80
			String[] splitStr = url.split("/");
			int len = splitStr.length;
			String tid = splitStr[len - 1];
			try {
				Thread.sleep((long) new Random().nextInt(5000));
			} catch (Exception e) {
			}
			
			if (!ProcessUtil.isNewArticle(tid, READ_FILE_NAME)) {
				System.out.println("文章已抓取过：" + url);
				return;
			}
			Document doc = ProcessUtil.getDoc(url);
			doc.setBaseUri(url);//
			Map<String, String> article = parseDoc(doc);
			if (article == null) {
				return;
			}
			ProcessUtil.postRsltMap(article);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	/***
	 * 解析DOCUMENT
	 * 
	 * @param doc
	 * @return
	 * @throws IOException
	 */
	public Map<String, String> parseDoc(Document doc) {
		Map<String, String> artcleMap = new HashMap<>();
		try {
			artcleMap.put("srcUrl", "" + doc.baseUri());
			String title = doc.title().replace("- 简书", "").replace("简书", "落事");
			artcleMap.put("title", title);
			Element bodyObj = doc.body();
			Elements content = bodyObj.select("div.show-content");
			artcleMap.put("summary", PreviewTextUtils.getText(content.text(), 126));
			// 替换所有超链接为空事件
			content.select("a[href]").attr("href", "javascript:void()");
			// 替换图片
			Elements imgObjs = content.select("img[src]");
			for (Element img : imgObjs) {
				img.attr("src", ProcessUtil.getNewImgUrl(img.attr("abs:src")));
			}
			artcleMap.put("content", content.toString());
			artcleMap.put("tags", ProcessUtil.getTag(title, content.text()));
			artcleMap.put("created", new Date() + "");
			ProcessUtil.saveHtmFile(content.toString());
		} catch (Exception e) {
			artcleMap.put("content", "管理员" + doc.baseUri());
			artcleMap.put("tags", "管理员");
			return artcleMap;
		}

		return artcleMap;
	}

	/**
	 * 记录当前读取的页面URL
	 * 
	 * @param url
	 * @returnd
	 */
	public Boolean nowPageUrl(String url) {
		String urla = "";
		String urlb = ProcessUtil.getNowPageUrl();
		if (!StringUtils.isBlank(urlb)) {
			urlb = urlb.substring(urlb.lastIndexOf("?") + 1, urlb.length());
			urlb = urlb.substring(0, urlb.lastIndexOf("page"));
			url = url.replaceAll(urlb, "");
			urla = url;
		} else {
			urla = url;
		}
		try {
			File f = new File(LUOSH_SPIDER_LOG + NOW_PAGE_URL_FILE);
			if (f.exists()) {
				f.delete();
			}
			f.createNewFile();
			// FileUtils.forceMkdir(f);
			FileUtils.writeStringToFile(f, urla + "\n", true);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return false;
	}

	public String getArcticleId(String url) {
		// http://bbs.51credit.com/forum.php?mod=viewthread&tid=1037839&extra=page%3D1%26orderby%3Dheats
		String[] pArr = url.split("&");
		for (String p : pArr) {
			if (p.startsWith("tid=")) {
				return p.replaceAll("tid=", "");
			}
		}
		return "-1";
	}

}
